#/usr/bin/env python
#-*- coding:utf-8 -*-
#Authot : will
# import os
# import sys
#
# import django
#
#
# sys.path.append('/Users/Macx/PycharmProjects/hk_news/')  # 将项目路径添加到系统搜寻路径当中
# os.environ['DJANGO_SETTINGS_MODULE'] = 'hk_news.settings'  # 设置项目的配置文件
#
# django.setup()  # 加载项目配置
#
from test01 import models



import requests
from bs4 import BeautifulSoup
from datetime import datetime

# 学有舍
# http://www.hyc.org.hk/data/



def open_url(url):
    req = requests.get(url)
    encoding = req.encoding
    # print(encoding)
    req = req.text.encode(encoding=encoding)
    soup =BeautifulSoup(req,'lxml')

    return soup




#-----------------------------------------------------
def xueyoushe():
    url ='http://www.hyc.org.hk/data/'
    soup = open_url(url)
    news=soup.select('.mediatablecss')[0]
    for row in news.select('tr')[1:]:
        # print(row.select('td'))
        for data in row.select('td'):
            updated_date=datetime.strptime(row.select('td')[0].text,'%Y年%m月%d日')
            url=row.select('td')[1].a.get('href').replace('..','http://www.hyc.org.hk').replace(' ','%20')
            title = row.select('td')[1].text.replace('\n','')

        if models.test.objects.filter(url=url,updated=str(updated_date),title=title,source='学友社').first() is None:
            models.test.objects.create(url=url,updated=str(updated_date),title=title,source='学友社')
        print(updated_date)

#-----------------------------------------------------
def student():
    url = 'http://student.hk/site/?q=news/108#news'
    soup = open_url(url)
    news=[]
    news_1 = soup.select('div.front_news h1')
    news_2 = soup.select('div.front_news h2')
    news_3 = soup.select('div.front_news h3')

    for i in news_1:news.append(i)
    for i in news_2:news.append(i)
    for i in news_3:news.append(i)
    news.pop(4)
    # print(news)
    # news.append(news_3)
    for row in news:
        # print(row)
        url='http://student.hk/site/'+row.a.get('href')
        big_title = row.text
        soup = open_url(url)
        for i in soup.select('div.inner_content a'):
            updated_date = (soup.select('h4 > span')[0].text.replace('\n','').split('				')[1])
            title =i.text
            url = i.get('href')
            if models.test.objects.filter(url=url,updated=str(updated_date),title=title,source='student.hk').first() is None:
                models.test.objects.create(url=url,updated=str(updated_date),title=title,source='student.hk')
            print(title,url)

    url = 'http://student.hk/site/?q=news/500'
    soup = open_url(url)
    news=[]
    news_1 = soup.select('div.front_news h1')
    news_2 = soup.select('div.front_news h2')
    news_3 = soup.select('div.front_news h3')

    for i in news_1:news.append(i)
    for i in news_2:news.append(i)
    for i in news_3:news.append(i)
    news.pop(4)

    for i in news:
        title=i.text
        url='http://student.hk/site/'+i.a.get('href')
        print(title,url)
        if models.test.objects.filter(url=url, title=title,
                                      source='student.hk').first() is None:
            models.test.objects.create(url=url,title=title,source='student.hk')

#-----------------------------------------------------

def hkdse():
    url='http://www.hkeaa.edu.hk/tc/HKDSE/info_corner/hkdse_feature_articles/'
    soup = open_url(url)
    for i in soup.select('div.inner_cont tr'):
        url = 'http://www.hkeaa.edu.hk/'+i.li.a.get('href')
        title=i.text.strip()
        if models.test.objects.filter(url=url, title=title,
                                      source='hkdse').first() is None:
            models.test.objects.create(url=url,title=title,source='hkdse')



hkdse()
student()
xueyoushe()